#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :基于item_id下载音频.py
# @Time      :2024/5/13 
# @Author    :CL
# @email     :1037654919@qq.com
import datetime
import json
import os
import re
import hashlib
import random
import secrets
import time
import requests
from retrying import retry
from util import mongo_manager, proxies  # proxies
from multiprocessing import Pool

xuexiqiangguo_item_id  = mongo_manager('xuexiqiangguo_item_id',db='public_data')

proxies = {
    'http': 'http://127.0.0.1:15732',
    'https': 'http://127.0.0.1:15732'
}
# 如果打开代理，则使用代理，注释掉下一行
# proxies = {}


def get_proxy():
    proxy = requests.get("http://47.120.55.234:5010/get/").json()
    proxies={"http": "http://{}".format(proxy)}.get("proxy")
    return proxies

def random_base36_string():
    # 生成一个随机的大整数
    random_number = secrets.randbelow(36 ** 11)  # 调整大小以匹配JS的长度
    # 转换为基数36的字符串
    return base36encode(random_number)


def base36encode(number):
    if not isinstance(number, int):
        raise TypeError('number must be an integer')

    if number < 0:
        raise ValueError('number must be positive')

    alphabet = '0123456789abcdefghijklmnopqrstuvwxyz'
    base36 = ''

    while number:
        number, i = divmod(number, 36)
        base36 = alphabet[i] + base36

    return base36 or alphabet[0]


def sha256_hash(string):
    # 创建一个 sha256 hash 对象
    hash_obj = hashlib.sha256()

    # 提供要加密的字符串，必须先将字符串转换成字节
    hash_obj.update(string.encode('utf-8'))

    # 获取十六进制格式的摘要
    hash_digest = hash_obj.hexdigest()

    return hash_digest

def get_token(article_id,url):
    # url = "https://boot-video.xuexi.cn/voice/12341945533476085762191005/p/ee33903560c7c64fc2a0884620821d12-c0de0de695fa4740a27ac0d4835a48cc-2.mp3"
    u = 103 # 固定
    timeStamp = int(time.time() * 1000)
    c = f'{article_id}_{random_base36_string()}_{str(timeStamp)}_{str(timeStamp+random.randint(1000, 3000))}'
    # c = "5073461499409858383_h14ywy18rjk_1715407206630_1715407214178"
    f_Y1 = "1d66f6b2-442a-4834-ba8c-c265e8769931" # 固定
    ciphertext = f"{url}_{u}_{c}_{f_Y1}"
    result = sha256_hash(ciphertext)
    return c,result

@retry(stop_max_attempt_number=3, wait_fixed=3000)
#基于id 获取音频链接
def get_mp3_url(item_id = '9056979967141844798',res_url="https://boot-video.xuexi.cn/voice/12377066921295585282191005/p/a842e33fc9b3085056db156dcf3dc36c-3c26367e0dde4166913fab8bf3162b02-2.mp3"):

    headers = {
        "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0",
        "Accept": "*/*",
        "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
        "Accept-Encoding": "gzip, deflate, br",
        "content-type": "application/json",
        "x-lwp-uid": "1715234954298883@pc-live",
        "Origin": "https://www.xuexi.cn",
        "Connection": "keep-alive",
        "Referer": "https://www.xuexi.cn/",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-site"
    }
    cookies = {
        "__UID__": "240797d0-0daf-11ef-801f-9b654f775548",
        "y-open-ua": "TorchApp%252F1.0%2520OSType%252FUbuntu%2520XueXi%252F2.19.0%2520Device%252Fxxqgpc%2520Ubuntu%252Fundefined(Ubuntu%253Bzh-CN)%2520lang%252Fzh-CN%2520pc.js%252F1706580237911%2520Scene%252Fpc",
        "y-open-did": "c51694c6aa91ed0bf155acef",
        "aliyungf_tc": "2864be1da263baaf8b24ea182f7e17af6f4aa8d49924331defcf8939fcd64bf0",
        "acw_tc": "ac11000117152349275265641ed796c8fe1782e8a93fa193d162d4c55568ba"
    }
    url = "https://gw-proxy-api.xuexi.cn/v1/api/exchangeAuthUrl"

    request_id,y_token = get_token(item_id,res_url)
    data = {
        "res_type": 103,
        "res_url": res_url,
        "request_id": request_id,
        "item_id": item_id,
        "y_uid": request_id,
        "y_token": y_token
    }
    data = json.dumps(data, separators=(',', ':'))
    response = requests.post(url, headers=headers, cookies=cookies, data=data,proxies=proxies)

    print(response.url,res_url,response)
    return response.json()
@retry(stop_max_attempt_number =3,wait_fixed=3000)
def callback(url = "https://boot-source.xuexi.cn/data/app/5073461499409858383.js"):
    headers = {
        "Accept": "*/*",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Pragma": "no-cache",
        "Referer": "https://www.xuexi.cn/",
        "Sec-Fetch-Dest": "script",
        "Sec-Fetch-Mode": "no-cors",
        "Sec-Fetch-Site": "same-site",
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
        "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Linux\""
    }
    cookies = {
        "__UID__": "15344de0-11ef-11ef-816a-d722d43974a4",
        "y-open-ua": "TorchApp%252F1.0%2520OSType%252FLinux%2520XueXi%252F2.19.0%2520Device%252Fxxqgpc%2520Linux%252Fx86_64(Linux%253Bzh-CN)%2520lang%252Fzh-CN%2520pc.js%252F1706580237911%2520Scene%252Fpc",
        "y-open-did": "72da71dda21bdf517dd93842"
    }
    # url = "https://boot-source.xuexi.cn/data/app/9056979967141844798.js"
    _st = int(time.time()*1000 +random.randint(1,999))
    params = {
        "callback": "callback",
        "_st": str(_st)
    }
    # proxies= get_proxy()
    response = requests.get(url, headers=headers, cookies=cookies, params=params,proxies=proxies,timeout=10)

    print(response.url,response)
    if response.status_code == 200:
        return response.text


def  run(item_id ,label ='思想' ): #基于item_id下载音频
    # item_id = '7990995120082897429'
    # path = f"/home/chenglei3/work/data/xuexiqiangguo/{label}/"
    today = datetime.datetime.now().strftime('%Y%m%d')
    path = f'/media/chenglei3/77D014CED257D1411/xuexiqiangguo_all/xuexiqiangguo_{today}/{label}/'
    os.makedirs(path, exist_ok=True)
    response_text = callback(url = f"https://boot-source.xuexi.cn/data/app/{item_id}.js")
    if response_text:
        # 找到第一个左括号和最后一个右括号的位置
        start_index = response_text.find('(')
        end_index = response_text.rfind(')')

        # 提取括号内的内容（JSON数据部分）
        json_data = response_text[start_index + 1:end_index]

        # 解析JSON数据
        data = json.loads(json_data)
        result={}
        if 'voices' in data and 'normalized_content' in data:
            result["_id"] = item_id
            result["label"] = label
            result['voices'] = data['voices']
            result['url'] = data['url']
            result['title'] = data['title']
            result['normalized_content'] = data['normalized_content']
            try:
                result['category'] = data['category']
                result['marking_tag'] = data['marking_tag']
                result['data_js_url'] = data['data_js_url']
                result['pc_url'] = data['pc_url']
            except:
                pass
            # 防止过长，特殊字符，文件保存失败

            title = result['title'].replace('|', '_').replace('：', '_').replace('!','_').replace('"',"_").replace(':',"_")
            # 使用正则表达式将特殊符号替换为下划线
            title_with_underscore = re.sub(r'[/\\:*?\"\<\>\|]', '_', title)
            if len(title) > 50:
                title = title[:50]
            try:
                xuexiqiangguo_item_id.updateOne({"_id":item_id}, result)
            except:
                print('插入失败')
            if result['voices']:
                if len(result['voices']) == 1:

                    url = result['voices'][0]['voice_storage_info'][0]['url']
                else:
                    for voice in result['voices']:
                        if voice['voice_meta_settings']['voice_language'] == 'mandarin':  # 普通话
                            url = voice['voice_storage_info'][0]['url']
                            break
                try:
                    down_mp3_url = get_mp3_url(item_id=str(item_id), res_url=url)['data']['auth_url']
                    response = requests.get(down_mp3_url,proxies=proxies,timeout=10)
                    if response.status_code == 200:
                        # 写入音频文件
                        with open(f'{path}/{title}.mp3', 'wb') as file:
                            file.write(response.content)

                        with open(f'{path}/{title}.txt', 'w', encoding='utf-8') as f:
                            f.write(title + '\n' + result['normalized_content'])
                    xuexiqiangguo_item_id.updateOne({"_id": item_id}, {"status": 'done','down_mp3_url':down_mp3_url})
                except:

                    xuexiqiangguo_item_id.updateOne({"_id": item_id}, {"status": 'error'})
            else:
                xuexiqiangguo_item_id.updateOne({"_id": item_id}, {"status": 'nodata'})
        else:
            xuexiqiangguo_item_id.updateOne({"_id": item_id}, {"status": 'nodata'})
    else:
        xuexiqiangguo_item_id.updateOne({"_id": item_id}, {"status": 'nodata'})
def main1():
    #栏目列表L
    labels=[
        {
            "label": "思 想",
            "url": "https://www.xuexi.cn/xxqg.html?id=e55ff0028ab0406e948cb0be9a8cae28",
            "key": "A"
        },
        {
            "label": "二十大时间",
            "url": "https://www.xuexi.cn/xxqg.html?id=a2543113741d4a2285f86f88f0afd87f"
        },
        {
            "label": "习近平文汇",
            "url": "https://www.xuexi.cn/5c90534c80d14c060d6683fa960e3676/82573c005c024095037d2186a02244cb.html",
            "key": "G"
        },
        {
            "label": "学习理论",
            "url": "https://www.xuexi.cn/xxqg.html?id=3cba33e067d64ded8a1a503f0774675c",
            "key": "C"
        },
        {
            "label": "红色中国",
            "url": "https://www.xuexi.cn/482f31e90c481f2161a19ecfcbb91f8b/e6109f421e472b4399b006ba21bc2ecd.html",
            "key": "D"
        },
        {
            "label": "学习科学",
            "url": "https://www.xuexi.cn/896bddc5f57a423b857a85eb40f98945/72742e3e40c96ade71e42b6e7ed42419.html",
            "key": "E"
        },
        {
            "label": "国 际",
            "url": "https://www.xuexi.cn/261c9a142ef8e6375ed554815a26d585/f2d8ff735982530b7a8c9bb90fa99f68.html",
            "key": "F"
        },
        {
            "label": "五个一工程",
            "url": "https://www.xuexi.cn/6dd69137d25db5673406cf8d1480548e/28d3e5cb24036a88cad7550485262507.html"
        },
        {
            "label": "学习电视台",
            "url": "https://www.xuexi.cn/0809b8b6ab8a81a4f55ce9cbefa16eff/ae60b027cb83715fd0eeb7bb2527e88b.html",
            "key": "H"
        },
        {
            "label": "学习电台",
            "url": "https://www.xuexi.cn/xxqg.html?id=e433dab0149a4d42bb914b7a296ea7ff",
            "key": "I"
        },
        {
            "label": "强军兴军",
            "url": "https://www.xuexi.cn/a9d25ec4ca996c6100d2982d33a1d8b2/4f02a12dd5663651422f5c55e10ef19e.html",
            "key": "L"
        },
        {
            "label": "学习文化",
            "url": "https://www.xuexi.cn/b069a45c80585e78b558c64306bfdf92/53194398355c363c5c635c74a70c7b45.html",
            "key": "K"
        }
    ]
    # 思想
    urllist = {
        # '思想': f"https://www.xuexi.cn/lgdata/e55ff0028ab0406e948cb0be9a8cae28.json?_st=28593108&js_v={int(time.time()) * 1000}",
        #         '二十大时间': f"https://www.xuexi.cn/lgdata/3cba33e067d64ded8a1a503f0774675c.json?_st=28594452&js_v={int(time.time()) * 1000}",
        #         '国际': f"https://www.xuexi.cn/lgdata/261c9a142ef8e6375ed554815a26d585/f2d8ff735982530b7a8c9bb90fa99f68.json?_st=28594464&js_v={int(time.time()) * 1000}",
        #         '强军兴军': f"https://www.xuexi.cn/lgdata/a9d25ec4ca996c6100d2982d33a1d8b2/4f02a12dd5663651422f5c55e10ef19e.json?_st=28594468&js_v={int(time.time()) * 1000}",
               }
    for label, url in urllist.items():

        text = str(get_lgdata(url = url))
        # 使用正则表达式查找链接及对应的item_id 能直接到item的链接形式
        links_with_item_id = re.findall(r'(https?://\S+)&item_id=(\d+)', text)

        # 打印找到的链接及对应的item_id
        for link, item_id in links_with_item_id:
            # print("Link:", link)

            if xuexiqiangguo_item_id.findOne({"item_id": item_id}) is None:
                print("Item ID:", item_id)
                run(item_id =item_id ,label =label)
            else:
                print("Item ID:", item_id, "已存在")

#获取频道列表
def  get_channel(url = "https://www.xuexi.cn/lgdata/channel-list.json"):
    headers = {
        "accept": "application/json",
        "accept-language": "zh-CN,zh;q=0.9",
        "cache-control": "no-cache",
        "pragma": "no-cache",
        "priority": "u=1, i",
        "referer": "https://www.xuexi.cn/lgpage/detail/index.html?id=13624761187570256842&item_id=13624761187570256842",
        "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Linux\"",
        "sec-fetch-dest": "empty",
        "sec-fetch-mode": "cors",
        "sec-fetch-site": "same-origin",
        "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
    }
    cookies = {
        "__UID__": "76cc1450-11ba-11ef-a76a-cda34ca49ac0",
        "_bl_uid": "ROlROwwC6ba0pCb64cRgp6U1Uqh4",
        "y-open-ua": "TorchApp%252F1.0%2520OSType%252FLinux%2520XueXi%252F2.19.0%2520Device%252Fxxqgpc%2520Linux%252Fx86_64(Linux%253Bzh-CN)%2520lang%252Fzh-CN%2520pc.js%252F1706580237911%2520Scene%252Fpc",
        "y-open-did": "619b1c38b09caeb3cb8acafc"
    }

    params = {
        "_st": "28594764",
        "js_v": "1706580237911"
    }
    response = requests.get(url, headers=headers, cookies=cookies, params=params)

    print(response.url,response)
    return response.json()
#获取item_id 基于总类
def get_lgdata(url =None):

    headers = {
        "Accept": "application/json",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Pragma": "no-cache",
        "Referer": "https://www.xuexi.cn/xxqg.html?id=e55ff0028ab0406e948cb0be9a8cae28",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-origin",
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
        "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Linux\""
    }
    cookies = {
        "__UID__": "31546700-10fd-11ef-aae7-c10cd7dd0de8"
    }
    if url is None:
        url = f"https://www.xuexi.cn/lgdata/e55ff0028ab0406e948cb0be9a8cae28.json?_st=28593108&js_v={int(time.time()) * 1000}"
        # params = {
        #     "_st": "28593108",
        #     "js_v": str(int(time.time()) * 1000)
        # }
    # response = requests.get(url, headers=headers, cookies=cookies, params=params)
    else:
        id_ = url.split('.json')[0].split('lgdata/')[-1]
        if '/' in id_:
            headers["Referer"] =str( 'https://www.xuexi.cn/' + id_ + '.html')
        else:
            headers["Referer"] =  f"https://www.xuexi.cn/xxqg.html?id={id_}"
    response = requests.get(url, headers=headers, cookies=cookies)

    print(response.url,response)
    return response.json()

def linshi():#临时数据库维护
    xuexiqiangguo_item_id.deleteMany({'voices': None})
    path = '/home/chenglei3/work/data/xuexiqiangguo_20240514/'
    for label in os.listdir(path):
        label_path = path + label + '/'
        print(label_path)
        for item_id in os.listdir(label_path):
            try:
                xuexiqiangguo_item_id.insertOne({"_id": item_id, 'label': label, 'item_id': item_id, 'status': 'done'})
            except Exception as e:
                print(e
                      )
                xuexiqiangguo_item_id.updateOne({"_id": item_id},
                                                {"_id": item_id, 'label': label, 'item_id': item_id, 'status': 'done'})

def downdataByitemid(num):
    print(f'这是第{num}进程')
    while True:
        seed =xuexiqiangguo_item_id.find_one_and_update({"status": None},{'status':'running'})
        if seed:
            run(item_id=seed['_id'],label=seed['label'])
        else:
            break
def main():
    pool = Pool(processes=5)
    numss = [i for i in range(1, 6)]
    pool.map(downdataByitemid, numss)
    pool.close()
    pool.join()


if __name__ == '__main__':
    print()
    # res= get_channel()
    # print(res)
    main()
    xuexiqiangguo_item_id.close()


